{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_files\n",
    "\n",
    "docs=load_files('clustering\\data')   #加载文件\n",
    "print(len(docs.data))\n",
    "print(len(docs.target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs.filenames[:6]   #filenames保存了所有文档的名字"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TF-IDF  统计学方法，用来评估一个词语对于一份文档的重要程度\n",
    "\n",
    "#### TF表示词频，特定的词语在文档中出现的次数除以文档中的词语总数\n",
    "\n",
    "#### IDF表示逆向文档频率指数，总文档的数目除以包含该词语的文档的数目，再对结果求对数，它表达的是词语的权重指数    IDF=log（1000/10）=2\n",
    "\n",
    "#### 词语对于文档的重要程度=TF*IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "max_features=20000\n",
    "\n",
    "vectorizer=TfidfVectorizer(\n",
    "    \n",
    "    max_df=0.4, # this, that, it如果一个词在40%的文档中出现过，认为是高频词，生成字典时会剔除它\n",
    "    \n",
    "    min_df=2, # 如果一个单词只在两个及两个以下的文档中出现过，则认为是低频词，生成字典时也剔除它\n",
    "    \n",
    "    max_features=max_features, #根据TF-IDF权重排序，取前20000\n",
    "    \n",
    "    encoding='latin-1'\n",
    "    \n",
    ")\n",
    "\n",
    "#TfidfVectorizer 把文档转换为矩阵，矩阵的每一行代表一个文档，一行中的每一个元素代表每一个词语的重要性\n",
    "\n",
    "X=vectorizer.fit_transform((d for d in docs.data))\n",
    "\n",
    "#fit_transform（）方法是fit（）和transform（）的结合。fit（）完成语料库分析、提取词典等操作\n",
    "\n",
    "# transform（）会把每篇文档转换成向量，最终构成一个矩阵\n",
    "\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "\n",
    "n_clusters=4\n",
    "kmean=KMeans(n_clusters=n_clusters,\n",
    "             max_iter=100,  #最多进行100次迭代\n",
    "             tol=0.01, #中心点移动距离小于0.01的时候，我们认为算法已经收敛，可以停止了\n",
    "             n_init=3,  #进行3次k-均值算法后求平均值\n",
    "             verbose=1   # verbose=1，则会输出迭代的过程信息           \n",
    "            )\n",
    "\n",
    "kmean.fit(X)\n",
    "\n",
    "print(kmean.score(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
